﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.IO;
using System.Threading;
using System.Collections;

namespace Spider
{
    public class DocumentWorker
    {
        private Uri m_uri;
        private Spider m_spider;
        private Thread[] m_thread;
        private int m_number;
        public const string IndexFile = "index.html";
        static object objectlink = new object();
        private HttpHelper httpHelper;

        public DocumentWorker(Spider spider)
        {
            httpHelper = new HttpHelper();
            m_spider = spider;
        }
        #region 本地创建下载的目录结构
        private string convertFilename(Uri uri) //分离HTTP地址，创建相应的目录结构
        {
            string result = SpiderAccount.DownloadPath;
            int index1;
            int index2;

            if (result[result.Length - 1] != '\\')
                result = result + "\\";

            String path = uri.PathAndQuery;  //得到域名后的字符串

            int lastSlash = path.LastIndexOf('/');  //最后一个/
            int lastDot = path.LastIndexOf('.');    //最后一个.

            if (path[path.Length - 1] != '/')
            {
                if (lastSlash > lastDot)
                    path += "/" + IndexFile;
            }

            lastSlash = path.LastIndexOf('/');

            string filename = "";
            if (lastSlash != -1)
            {
                filename = path.Substring(1 + lastSlash);  //取得url后面的文件名及以后的
                path = path.Substring(0, 1 + lastSlash);  //取得文件名以前的
                filename = filename.Replace(".html", ".txt");
                if (filename.Equals(""))  //文件名为空的话，就设成index.html
                {
                    filename = IndexFile;
                }
            }
            // 必要时创建目录结构			
            index1 = 1;
            do
            {
                index2 = path.IndexOf('/', index1);   //获得目录的第二个/
                if (index2 != -1)
                {
                    String dirpart = path.Substring(index1, index2 - index1);
                    result += dirpart; //得到两个/之间的
                    result += "\\";

                    //try
                    //{
                    Directory.CreateDirectory(result);
                    //}
                    //catch
                    //{
                    //    return "";
                    //}
                    index1 = index2 + 1;
                }
            } while (index2 != -1);

            result += filename;
            Log.WriteLog(@Environment.CurrentDirectory + "\\url文件名.txt", result);
            return result;
        }
        #endregion

        #region 保存二进制文件
        private void SaveBinaryFile(WebResponse response) //存储二进制文件，二进制文件的内容类型声明不以"text/"开头
        {
            byte[] buffer = new byte[1024];

            string filename = convertFilename(response.ResponseUri);

            if (filename.Contains("?"))
            {
                filename = filename.Replace("?", "").Replace(":", "");
            }
            if (filename == "")
                return;
            Stream outStream = File.Create(filename);  //确定了输出文件的名字和路径之后就可以打开读取Web页面的输入流、写入本地文件的输出流
            Stream inStream = response.GetResponseStream();

            int l;  //接下来就可以读取Web文件的内容并写入到本地文件，这可以通过一个循环方便地完成
            do
            {
                l = inStream.Read(buffer, 0, buffer.Length);
                if (l > 0)
                {
                    outStream.Write(buffer, 0, l);
                }
            }
            while (l > 0);
            outStream.Close();
            inStream.Close();
        }
        private void SaveBinaryFile(string url, string path) //存储二进制文件，二进制文件的内容类型声明不以"text/"开头
        {
            try
            {
                string fileextension = url.Substring(url.LastIndexOf('.') + 1);
                if (!SpiderAccount.AllowURLType.Contains(fileextension))  //如果此链接的扩展名不在允许的列表里，就不下载，防止下载aspx等
                {
                    return;
                }
                HttpWebResponse response = httpHelper.GetHtmlResponse(url, CookieMng.CurrentCookie); //HttpWebRequest类是来下载HTML页面

                byte[] buffer = new byte[1024];

                string filename = path;

                if (filename.Contains("?"))
                {
                    filename = filename.Replace("?", "").Replace(":", "");
                }
                if (filename == "")
                    return;
                if (response != null)
                {
                    Stream outStream = File.Create(filename);  //确定了输出文件的名字和路径之后就可以打开读取Web页面的输入流、写入本地文件的输出流
                    Stream inStream = response.GetResponseStream();

                    int l;  //接下来就可以读取Web文件的内容并写入到本地文件，这可以通过一个循环方便地完成
                    do
                    {
                        l = inStream.Read(buffer, 0, buffer.Length);
                        if (l > 0)
                        {
                            outStream.Write(buffer, 0, l);
                        }
                    }
                    while (l > 0);
                    outStream.Close();
                    inStream.Close();
                }
            }
            catch (Exception err)
            {
                Log.LogSpecial(LogDetail.LogType.GoloalRecord, err.Message);
            }
        }

        #endregion

        #region 保存文本文档
        private void SaveTextFile(string buffer, string url) //文本文件的内容类型总是以"text/"开头，假设文件已被下载并保存到了一个字符串，这个字符串可以用来分析网页包含的链接，当然也可以保存为磁盘上的文件。     
        {
            try
            {
                //lock (objectlink)
                //{
                SpiderAccount.TimerCount++;
                int fileoffset = SpiderAccount.TimerCount;//用来标识文件
                buffer = buffer.ToLower();
                #region 取标题，来做文件名
                string title = Utility.Resove(buffer, "<title>", "</title>");
                if (string.IsNullOrEmpty(title))
                {
                    title = title.Replace(" ", "");
                    title = title.Length > 15 ? title.Substring(0, 15) : title;
                }
                #endregion

                if (SpiderAccount.StartStr != string.Empty || SpiderAccount.EndStr != string.Empty)
                {
                    buffer = Utility.Resove(buffer, SpiderAccount.StartStr, SpiderAccount.EndStr);
                    if (string.IsNullOrEmpty(buffer))  //没有取到所要的字符串的话，就不保存
                        return;
                }
                m_spider.ShowUrl(fileoffset, url, title, false);
                title = fileoffset.ToString() + title;  //title是文件夹和文件的名字，加上随机数防止重复，有可能一个网站里titile里的值都是一样的
                title = Utility.ReplaceBadCharOfFileName(title);   //过滤title标签里的非法字符

                string directory = SpiderAccount.DownloadPath + "\\" + title; //每个新闻对应一个文件夹，此文件夹用来保存新闻的图片
                Directory.CreateDirectory(directory);

                string filename = SpiderAccount.DownloadPath + "\\" + title + ".txt";

                IList<string> list = ParseHtmlNew.GetHyperLinkByParse(buffer);  //取得内容里所有的链接
                foreach (string s in list)
                {
                    string link = s;
                    HandleSrcAndHref(ref link, new Uri(url));  //处理链接
                    string linkname = link.Substring(link.LastIndexOf('/') + 1);  //取得链接的文件名
                    if (link.Contains("?"))
                    {
                        linkname = linkname.Split('?')[0].Substring(linkname.LastIndexOf('/') + 1);
                    }
                    SaveBinaryFile(link, directory + "\\" + linkname);  //保存此内容里附带的图片
                }

                #region -------------------------内容页分页的处理------------------------------------------
                HttpWebResponse response = httpHelper.GetHtmlResponse(url, CookieMng.CurrentCookie); //请求第一页
                if (response != null)
                {
                    Stream stream = response.GetResponseStream();

                    string temp = new StreamReader(stream, httpHelper.Encoding).ReadToEnd();
                    IList<string> listcontent = ParseHtmlNew.GetHyperLinkByParse(temp);//得到第一个所有的url

                    System.Text.RegularExpressions.Regex regex = new System.Text.RegularExpressions.Regex(@SpiderAccount.ContentUrlReg.ToLower());
                    foreach (string page in listcontent)
                    {
                        if (regex.IsMatch(page))
                        {
                            string link = page;
                            HandleSrcAndHref(ref link, url);  //处理链接

                            if (link == url)  //第一页不再处理
                            {
                                continue;
                            }
                            #region  下载内容页
                            HttpWebResponse responsecontent = httpHelper.GetHtmlResponse(link, CookieMng.CurrentCookie);
                            if (responsecontent == null)
                            {
                                continue;
                            }
                            if (!responsecontent.ContentType.ToLower().StartsWith("text/"))  //确定该文件是否为二进制文件
                            {
                                continue;
                            }
                            string tempstr = "";

                            Stream streams = responsecontent.GetResponseStream();
                            StreamReader reader = new StreamReader(streams, httpHelper.Encoding); //如果该文件是文本文件，首先从Stream创建一个StreamReader，然后将文本文件的内容一行一行读取出来
                            tempstr = reader.ReadToEnd();
                            #region 下载图片
                            IList<string> listjpg = ParseHtmlNew.GetHyperLinkByParse(tempstr);  //取得内容里所有的链接
                            foreach (string s in listjpg)
                            {
                                link = s;
                                HandleSrcAndHref(ref link, new Uri(url));  //处理链接
                                string linkname = link.Substring(link.LastIndexOf('/') + 1);  //取得链接的文件名
                                if (link.Contains("?"))
                                {
                                    linkname = linkname.Split('?')[0].Substring(linkname.LastIndexOf('/') + 1);
                                }
                                SaveBinaryFile(link, directory + "\\" + linkname);  //保存此内容里附带的图片
                            }
                            lock (objectlink)
                            {
                                m_spider.m_urlCount++;
                            }
                            #endregion
                            tempstr = tempstr.ToLower();
                            tempstr = Utility.Resove(tempstr, SpiderAccount.StartStr, SpiderAccount.EndStr);
                            tempstr = Utility.ReplaceHtmlCharacter(tempstr);
                            buffer += tempstr;
                            #endregion
                        }
                    }
                }

                #endregion

                #region 替换html字符
                buffer = Utility.ReplaceHtmlCharacter(buffer);
                #endregion

                if (filename == "")
                    return;

                StreamWriter outStream = new StreamWriter(filename, false, httpHelper.Encoding);//在这里，我们首先打开一个文件输出流，然后将字符串内容写入流，保存到文件里，最后关闭文件
                outStream.Write(buffer);
                outStream.Close();
                //}
            }
            catch (Exception err)
            {
                Log.LogSpecial(LogDetail.LogType.GoloalRecord, err.Message);
            }
        }
        #endregion

        #region 下载一个url
        private string GetPage()  //下载一个文本文档或是二进制文件（图片）
        {
            HttpWebResponse response = null;
            Stream stream = null;
            StreamReader reader = null;
            try
            {
                response = httpHelper.GetHtmlResponse(m_uri.ToString(), CookieMng.CurrentCookie); //HttpWebRequest类是来下载HTML页面
                if (response == null)
                {
                    return "";
                }
                stream = response.GetResponseStream();


                if (!response.ContentType.ToLower().StartsWith("text/"))  //确定该文件是否为二进制文件
                {
                    //SaveBinaryFile(response);
                    return null;  //不要图片，直接返回
                }
                string buffer = "", line;
                reader = new StreamReader(stream, httpHelper.Encoding); //如果该文件是文本文件，首先从Stream创建一个StreamReader，然后将文本文件的内容一行一行读取出来
                //while ((line = reader.ReadLine()) != null)
                //{
                //    buffer += line + "\r\n";
                //}
                buffer = reader.ReadToEnd();
                lock (objectlink)
                {
                    m_spider.m_urlCount++;
                }
                buffer = buffer.ToLower();
                string title = Utility.Resove(buffer, "<title>", "</title>");
                m_spider.ShowUrl(0, m_uri.ToString(), title, false);

                IList<string> list = ParseHtmlNew.GetHyperLinkByParse(buffer);  //取得列表里的所有链接
                foreach (string s in list)
                {
                    System.Text.RegularExpressions.Regex regex = new System.Text.RegularExpressions.Regex(@SpiderAccount.ListUrlReg.ToLower());
                    if (regex.IsMatch(s))
                    {
                        string link = s;
                        HandleSrcAndHref(ref link, SpiderAccount.Url);  //处理链接
                        //string linkname = link.Substring(link.LastIndexOf('/') + 1);  //取得链接的文件名
                        //if (link.Contains("?"))
                        //{
                        //    linkname = link.Split('?')[0].Substring(link.LastIndexOf('/') + 1);
                        //}

                        #region  下载内容页

                        response = httpHelper.GetHtmlResponse(link, CookieMng.CurrentCookie); //HttpWebRequest类是来下载HTML页面
                        if (response == null)
                        {
                            return "";
                        }
                        stream = response.GetResponseStream();

                        if (!response.ContentType.ToLower().StartsWith("text/"))  //确定该文件是否为二进制文件
                        {
                            //SaveBinaryFile(response);
                            return null;  //不要图片，直接返回
                        }
                        buffer = "";
                        line = "";
                        reader = new StreamReader(stream, httpHelper.Encoding); //如果该文件是文本文件，首先从Stream创建一个StreamReader，然后将文本文件的内容一行一行读取出来
                        //while ((line = reader.ReadLine()) != null)
                        //{
                        //    buffer += line + "\r\n";
                        //}
                        buffer = reader.ReadToEnd();
                        lock (objectlink)
                        {
                            m_spider.m_urlCount++;
                        }
                        #endregion
                        SaveTextFile(buffer, link);
                    }
                }
                //buffer = reader.ReadToEnd();
                return "";
            }
            catch (WebException e)
            {
                lock (objectlink)
                {
                    m_spider.m_urlFailCount++;
                }
                System.Console.WriteLine("下载失败，错误：" + e);
                Log.LogSpecial(LogDetail.LogType.GoloalRecord, e.Message);

                m_spider.ShowUrl(0, m_uri.ToString(), "", true);
                return null;
            }
            catch (IOException e)
            {
                lock (objectlink)
                {
                    m_spider.m_urlFailCount++;
                }
                System.Console.WriteLine("下载失败，错误：" + e);
                Log.LogSpecial(LogDetail.LogType.GoloalRecord, e.Message);
                m_spider.ShowUrl(0, m_uri.ToString(), "", true);
                return null;
            }
            catch (Exception e)
            {
                lock (objectlink)
                {
                    m_spider.m_urlFailCount++;
                }
                Log.LogSpecial(LogDetail.LogType.GoloalRecord, e.Message);
                m_spider.ShowUrl(0, m_uri.ToString(), "", true);
                return null;
            }
            finally
            {
                if (reader != null) reader.Close();
                if (stream != null) stream.Close();
                if (response != null) response.Close();
            }
        }
        #endregion

        #region 把解析后的url加入到队列中
        private void ProcessLink(string link)
        {
            Uri url;
            try
            {
                m_uri = new Uri(link);
                if (m_uri.Host != SpiderAccount.UrlHost)
                {
                    #region 验证非本域名下的URL
                    int k = link.LastIndexOf('.');
                    string outurl = link.Substring(k + 1);  //取后缀名
                    if (!SpiderAccount.AllowURLType.Contains(outurl))
                    {
                        Log.LogSpecial(LogDetail.LogType.AllOutUrl, m_uri.ToString());
                        return;
                    }
                    #endregion
                }
                else
                {
                    #region 下载指定目录下的页面
                    int k = link.LastIndexOf('.');
                    string outurl = link.Substring(k + 1);  //取后缀名
                    if (!SpiderAccount.AllowURLType.Contains(outurl))
                    {
                        string contain = SpiderAccount.UrlContain;
                        if (!string.IsNullOrEmpty(contain))
                        {
                            if (!link.Contains(contain))
                            {
                                return;
                            }
                        }
                    }
                    #endregion
                }
                url = m_uri;
            }
            catch (UriFormatException e)
            {
                //System.Console.WriteLine("Invalid URI:" + link + " Error:" + e.Message);
                //return;
                Log.WriteLog(e.Message);
                return;
            }
            if (!url.Scheme.ToLower().Equals("http") && !url.Scheme.ToLower().Equals("https"))
            {
                return;
            }
            if (!url.Host.ToLower().Equals(m_uri.Host.ToLower()))  //一个host下进行搜索
            {
                return;
            }
            m_spider.addURI(url);  //加入到队列中，是否重复会自动判断
        }
        #endregion

        #region 处理已经下载过的text类型的文档
        private void ProcessPage(string page, Uri uri)
        {
            try
            {
                string content = page;
                //IList<string> list = ParseHtmlNew.GetHyperLink(page);
                IList<string> list = ParseHtmlNew.GetHyperLinkByParse(page);
                foreach (string s in list)
                {
                    string link = s;
                    HandleSrcAndHref(ref link, uri);
                    ProcessLink(link);
                }
            }
            catch (Exception err)
            {
                Log.LogSpecial(LogDetail.LogType.GoloalRecord, err.Message);
            }
        }
        #endregion

        #region  处理src或href上面的URL
        public void HandleSrcAndHref(ref string link, Uri uri)
        {
            try
            {
                #region 以前的
                //Log.WriteLog(@Environment.CurrentDirectory + "\\Url处理.txt", link + "----------" + uri.ToString());
                string url = uri.ToString();
                url = url.Remove(url.LastIndexOf("/"));   //移除后面的文件名
                if (link.IndexOf('/') == 0)  //相对根目录的路径
                {
                    link = SpiderAccount.Url + link;
                }
                else if (link.Contains(".."))  //如果链接中含有..
                {
                    int count = 0; //i表示的是..的数量
                    int i = link.Length;
                    do
                    {
                        int j = link.LastIndexOf("..", i);
                        if (j >= 0)
                        {
                            count++;
                            link = link.Remove(j, 2);   //移除..
                            url = url.Remove(url.LastIndexOf("/"));

                            i = j;
                        }
                    }
                    while (i > 0);

                    link = url + link;
                }
                else if (!link.Contains("http:"))
                {
                    link = url.ToString() + "/" + link;
                }
                #endregion
            }
            catch (Exception err)
            {
                Log.LogSpecial(LogDetail.LogType.GoloalRecord, err.Message);
            }
        }
        public void HandleSrcAndHref(ref string link, string uri)
        {
            int k = link.IndexOf('/');  //  /表示根目录
            if (k == 0)
            {
                link = "http://" + SpiderAccount.UrlHost + "/" + link;
                return;
            }
            if (!link.Contains("http://"))
            {
                int i = uri.LastIndexOf('/');
                link = uri.Substring(0, i) + "/" + link;
            }
        }
        #endregion

        #region 获取一个可用的url，并下载它，不可用的话，将一直等待（ObtainWork的作用）
        public void Process()
        //多线程使得计算机看起来就像能够同时执行一个以上的操作，不过，除非计算机包含多个处理器，否则，所谓的同时执行多个
        //操作仅仅是一种模拟出来的效果。靠计算机在多个线程之间快速切换达到”同时“执行多个操作的效果。一般而言，只有在两种
        //情况下才能事实上提高程序运行的速度。
        //第一种情况使计算机拥有多个处理器，第二种情况是程序经常要等待某个外部事件。
        //为此，我们用DocumentWorker类封装所有下载一个URL的操作。每当一个DocumentWorker的实例被创建，它就进入循环，等待下一个要处理的URL。
        {
            while (!m_spider.Quit) //这个循环将一直运行，直至Quit标记被设置成了true（当用户点击"Cancel"按钮时，Quit标记就被设置成true）。
            {
                try
                {
                    m_uri = m_spider.ObtainWork();  //获取一个URL，ObtainWork将一直等待，直到有一个URL可有，这要有其它线程解析文档并寻找链接才能获得

                    m_spider.SpiderDone.WorkerBegin();
                    string num = "Download(" + Thread.CurrentThread.Name + "):" + m_uri;

                    Log.WriteLog(num); //记下线程下载的url
                    string page = GetPage();  //得到的是一个字符串形式的html文档
                    if (page != null)
                    {
                        //ProcessPage(page, m_uri);  //传递一个本页面的url，可以帮助正确处理url路径问题
                    }
                    m_spider.SpiderDone.WorkerEnd();
                }
                catch (Exception err)
                {
                    System.Windows.Forms.MessageBox.Show(err.Message);
                    Log.LogSpecial(LogDetail.LogType.GoloalRecord, err.Message);
                }
            }
        }
        #endregion

        /// <summary>
        /// 开始线程
        /// </summary>
        public void StartThread()  //开始处理url
        {
            if (m_thread != null && m_thread.Length > 0)  //未解决暂停问题
            {
                foreach (Thread thread in m_thread)
                {
                    if (thread.ThreadState == ThreadState.Suspended)
                        thread.Resume();
                }
                return;
            }
            //ThreadStart ts = new ThreadStart(this.Process);
            m_thread = new Thread[SpiderAccount.ThreadCount];
            for (int i = 0; i < SpiderAccount.ThreadCount; i++)
            {
                m_thread[i] = new Thread(new ThreadStart(this.Process));
                m_thread[i].Name = "T" + i.ToString();
                m_thread[i].Priority = ThreadPriority.Highest;
                m_thread[i].IsBackground = true;  //设置后台线程
                m_thread[i].Start();
            }
        }
        /// <summary>
        /// 终止线程
        /// </summary>
        public void StopThread()
        {
            try
            {
                foreach (Thread thread in m_thread)
                {
                    thread.Suspend();  //终止线程时，不彻底
                }
            }
            catch (Exception err)
            {
                //System.Windows.Forms.MessageBox.Show(err.Message);
            }
        }
        public int Number  //用来标识线程的
        {
            get { return m_number; }
            set { m_number = value; }
        }
    }
}
